﻿using System;
using System.Collections.Generic;
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Diagnostics;

/// <summary>
/// 育儿网 抓取逻辑
/// </summary>
public static class Ci123
{
    /// <summary>
    /// 搜索列表抓取
    /// </summary>
    /// <param name="url">起始url</param>
    /// <param name="ispage">是否需要翻页抓取</param>
    /// <returns></returns>
    public static string GetSerach(string url, bool ispage = true)
    {
        string res = string.Empty;



        return res;
    }


    public static string Quest(string url)
    {
        string msg = string.Empty;
        try
        {
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
            req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, sdch");
            req.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en;q=0.6");
            req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
            WebResponse resp = req.GetResponse();
            StreamReader read = new StreamReader(resp.GetResponseStream(), Encoding.UTF8);
            msg = read.ReadToEnd();
            AllClose(read, resp);
        }
        catch (Exception)
        {

            throw;
        }
        return msg;
    }

    public static string getDataFromUrl(string url)
    {
        string str = string.Empty;
        HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

        //设置http头  
        request.AllowAutoRedirect = true;
        request.AllowWriteStreamBuffering = true;
        request.Referer = "http://search.ci123.com/all/%E6%B9%BF%E7%96%B9.html?p=1";
        request.Timeout = 10 * 1000;
        request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";

        HttpWebResponse response = null;
        try
        {
            response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK)
            {
                //根据http应答的http头来判断编码  
                string characterSet = response.CharacterSet;
                Encoding encode;
                if (characterSet != "")
                {
                    if (characterSet == "ISO-8859-1")
                    {
                        characterSet = "gb2312";
                    }
                    else if (characterSet == "UTF8")
                    {
                        characterSet = "UTF-8";
                    }
                    encode = Encoding.GetEncoding(characterSet);
                }
                else
                {
                    encode = Encoding.Default;
                }

                //声明一个内存流来保存http应答流  
                Stream receiveStream = response.GetResponseStream();
                MemoryStream mStream = new MemoryStream();

                byte[] bf = new byte[255];
                int count = receiveStream.Read(bf, 0, 255);
                while (count > 0)
                {
                    mStream.Write(bf, 0, count);
                    count = receiveStream.Read(bf, 0, 255);
                }
                receiveStream.Close();

                mStream.Seek(0, SeekOrigin.Begin);

                //从内存流里读取字符串  
                StreamReader reader = new StreamReader(mStream, encode);
                char[] buffer = new char[1024];
                count = reader.Read(buffer, 0, 1024);
                while (count > 0)
                {
                    str += new String(buffer, 0, count);
                    count = reader.Read(buffer, 0, 1024);
                }

                //从解析出的字符串里判断charset，如果和http应答的编码不一直  
                //那么以页面声明的为准，再次从内存流里重新读取文本  
                Regex reg =
                    new Regex(@"<meta[\s\S]+?charset=(.*)""[\s\S]+?>",
                              RegexOptions.Multiline | RegexOptions.IgnoreCase);
                MatchCollection mc = reg.Matches(str);
                if (mc.Count > 0)
                {
                    string tempCharSet = mc[0].Result("$1");
                    if (string.Compare(tempCharSet, characterSet, true) != 0)
                    {
                        encode = Encoding.GetEncoding(tempCharSet);
                        str = string.Empty;
                        mStream.Seek(0, SeekOrigin.Begin);
                        reader = new StreamReader(mStream, encode);
                        buffer = new char[255];
                        count = reader.Read(buffer, 0, 255);
                        while (count > 0)
                        {
                            str += new String(buffer, 0, count);
                            count = reader.Read(buffer, 0, 255);
                        }
                    }
                }
                reader.Close();
                mStream.Close();
            }
        }
        catch (Exception ex)
        {
            Trace.TraceError(ex.ToString());
        }
        finally
        {
            if (response != null)
                response.Close();
        }
        return str;
    }

    public static void AllClose(StreamReader sr, WebResponse rp)
    {
        sr.Close();
        sr.Dispose();
        rp.Close();
    }
}